#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# @Time    : 2018/2/2 0002 11:44
# @Author  : Arliki
# @email   : hkdnxycz@outlook.com
# @File    : get_ip

import redis, time, random, scrapy
import requests as rq

from scrapy_redis.spiders import RedisSpider
from qqcoment.items import GetipItem

page = 4


class XiciSpider(RedisSpider):
    name = 'xici'
    redis_key = 'ip:start_url'
    custom_settings = {
        'REDIS_PARAMS': {
            'db': 3,
            'password': '5tgbnhy67ujm'
        }
    }

    def __init__(self, *args, **kwargs):
        domain = kwargs.pop('domain', '')
        self.allowed_domains = filter(None, domain.split(','))
        super(XiciSpider, self).__init__(*args, **kwargs)

    def parse(self, response):
        global page
        ips = response.xpath('//tr[@class="odd"]')
        for ip in ips:
            item = dict()
            item['host'] = ip.xpath('td/text()')[0].extract()
            item['port'] = ip.xpath('td/text()')[1].extract()
            for i in range(2, 8):
                if ip.xpath('td/text()')[i].extract().lower() == 'http' or ip.xpath('td/text()')[
                    i].extract().lower() == 'https':
                    item['protocol'] = ip.xpath('td/text()')[i].extract().lower()
                    break
            yield item
        if page <= 7:
            time.sleep(2)
            page += 1
            url = "http://www.xicidaili.com/nn/%d" % page
            yield scrapy.Request(url, self.parse, dont_filter=True)



            # lpush ip:start_url http://www.xicidaili.com/nn/1
